import json
import traceback

from selenium import webdriver  # 从selenium导入webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import time

base = 'https://www.webofscience.com'
caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},  # 记录性能日志
}

driverPath = './chromedriver.exe'
driver = webdriver.Chrome(executable_path=driverPath, desired_capabilities=caps)
driver.get(base)
urls = set()
driver.implicitly_wait(20)

input("搜索结果页面加载完毕后请按下回车：")
logs = driver.get_log("performance")
driver.refresh()

isNextPage = True


def GetUrls():
    logs = driver.get_log("performance")
    for log in logs:
        try:
            imess = json.loads(log['message'])['message']

            if imess['method'] == 'Network.webSocketFrameReceived':
                isData = False
                if imess['params']['response']['payloadData'][0] == '{':
                    imess = json.loads(imess['params']['response']['payloadData'])
                    isData = True
                if isData and imess['key'] == 'records':
                    for item in imess['payload'].values():
                        urls.add("https://www.webofscience.com/wos/alldb/full-record/{}".format(item['colluid']))
        except:
            continue


def scrollToBottom():
    appRecords = driver.find_elements(by=By.XPATH, value='//app-record')
    driver.execute_script(
        "Object.defineProperty(document, 'visibilityState', {value: 'visible', writable: true});Object.defineProperty(document, 'hidden', {value: false, writable: true});document.dispatchEvent(new Event('visibilitychange'));")
    driver.execute_script("arguments[0].scrollIntoView();", appRecords[-1])


def ClickNextPage():
    global isNextPage
    nextPage = driver.find_element(
        by=By.XPATH, value="//app-page-controls//form//button[@cdxanalyticscategory='wos_navigation_next_page']")
    isNextPage = not nextPage.get_property("disabled")
    if isNextPage:
        driver.execute_script("arguments[0].click();", nextPage)


while isNextPage:
    try:
        element = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.XPATH, '//app-records-list/app-record//app-summary-title//a'))
        )
        GetUrls()
        # scrollToBottom()
        ClickNextPage()
        # time.sleep(1)
    except TimeoutException:
        time.sleep(420)
        logs = driver.get_log("performance")
        driver.refresh()
        continue
    except Exception as e:
        traceback.print_exc()
        break

with open("urls.json", 'w') as f:
    f.write(json.dumps(list(urls)))
